//==============================================================================
//
//     DObjcTokenizer - the Objective-C tokenizer class in the ofc-library
//
//               Copyright (C) 2005  Dick van Oudheusden
//  
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public
// License along with this library; if not, write to the Free
// Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
//
//==============================================================================
// 
//  $Date: 2005-10-16 05:53:25 $ $Revision: 1.13 $
//
//==============================================================================

#include "ofc/DObjcTokenizer.h"

#include <ctype.h>


#if _INTERFACE_

#include "ofc/config.h"

#include "ofc/DSource.h"

#include "ofc/DText.h"
#include "ofc/DList.h"

#include "ofc/DTextReadable.h"


// General tokens
#define DOT_UNKNOWN          (100) // Unknown token
#define DOT_EOFF             (101) // End Of File
#define DOT_EOFL             (102) // End Of Line
#define DOT_WHITESPACE       (103) // White space
#define DOT_COMMENT          (104) // Comment
#define DOT_IDENTIFIER       (105) // Identifier
#define DOT_OPERATOR         (106) // General operator
#define DOT_PREPROCESSOR     (107) // General Preprocessor

// Constants
#define DOT_DEC_NUMBER       (200) // Decimal number
#define DOT_OCT_NUMBER       (201) // Octal number
#define DOT_HEX_NUMBER       (202) // Hex number
#define DOT_FLP_NUMBER       (203) // Floating point number
#define DOT_CHAR             (204) // Character constant
#define DOT_STRING           (205) // String
#define DOT_WIDE_CHAR        (206) // Wide character constant
#define DOT_WIDE_STRING      (207) // Wide string
#define DOT_OBJC_STRING      (208) // Objective-c string

// Preprocessor subtokens
#define DOT_PASSERT          (300) // the assert directive
#define DOT_PDEFINE          (301) // the define directive
#define DOT_PELIF            (302) // the else if directive
#define DOT_PELSE            (303) // the else directive
#define DOT_PENDIF           (304) // the endif directive
#define DOT_PERROR           (305) // the error directive
#define DOT_PIDENT           (306) // the ident directive
#define DOT_PIF              (307) // the if directive
#define DOT_PIFDEF           (308) // the ifdef directive
#define DOT_PIFNDEF          (309) // the ifndef directive
#define DOT_PINCLUDE         (310) // the file include directive
#define DOT_PIMPORT          (311) // the import directive
#define DOT_PLINE            (312) // the line directive
#define DOT_PPRAGMA          (313) // the pragma directive
#define DOT_PUNASSERT        (314) // the unassert directive
#define DOT_PUNDEF           (315) // the undefine directive
#define DOT_PWARNING         (316) // the warning directive

// Predefined types subtokens
#define DOT_TCHAR            (400) // the char type
#define DOT_TDOUBLE          (401) // the double type
#define DOT_TFLOAT           (402) // the float type
#define DOT_TINT             (403) // the integer type
#define DOT_TSHORT           (404) // the short type
#define DOT_TLONG            (405) // the long type
#define DOT_TUNSIGNED        (406) // the unsigned type
#define DOT_TVOID            (407) // the void type
#define DOT_TSIGNED          (408) // the signed type

// Storage subtokens
#define DOT_AUTO             (450) // the auto keyword
#define DOT_CONST            (451) // the const keyword
#define DOT_EXTERN           (452) // the extern keyword
#define DOT_REGISTER         (453) // the register keyword
#define DOT_STATIC           (454) // the static keyword
#define DOT_VOLATILE         (455) // the volatile keyword

// Type definition subtokens
#define DOT_ENUM             (480) // the enum keyword
#define DOT_STRUCT           (481) // the struct keyword
#define DOT_TYPEDEF          (482) // the typedef keyword
#define DOT_UNION            (483) // the union keyword

// Flow subtokens
#define DOT_BREAK            (500) // the break keyword
#define DOT_CASE             (501) // the case keyword
#define DOT_CONTINUE         (502) // the continue keyword
#define DOT_DEFAULT          (503) // the default keyword
#define DOT_DO               (504) // the do keyword
#define DOT_ELSE             (505) // the else keyword
#define DOT_FOR              (506) // the for keyword
#define DOT_GOTO             (507) // the goto keyword
#define DOT_IF               (508) // the if keyword
#define DOT_RETURN           (509) // the return keyword
#define DOT_SWITCH           (510) // the switch keyword
#define DOT_WHILE            (511) // the while keyword

// Objective-C keywords
#define DOT_INTERFACE        (530) // the @interface keyword
#define DOT_IMPLEMENTATION   (531) // the @implementation keyword
#define DOT_PROTOCOL         (532) // the @protocol keyword
#define DOT_END              (533) // the @end keyword
#define DOT_PRIVATE          (534) // the @private keyword
#define DOT_PROTECTED        (535) // the @protected keyword
#define DOT_PUBLIC           (536) // the @public keyword
#define DOT_SELECTOR         (537) // the @selector keyword
#define DOT_CLASS            (538) // the @class keyword
#define DOT_ENCODE           (539) // the @encode keyword
#define DOT_DEFS             (540) // the @defs keyword
#define DOT_TRY              (541) // the @try keyword
#define DOT_CATCH            (542) // the @catch keyword
#define DOT_FINALLY          (543) // the @finally keyword
#define DOT_THROW            (544) // the @throw keyword
#define DOT_SYNCHRONIZED     (545) // the @synchronized keyword
#define DOT_SYNCHRONIZE      (546) // the @synchronize keyword

// Logical operators
#define DOT_AND              (601) // the && operator
#define DOT_OR               (602) // the || operator
#define DOT_NOT              (603) // the ! operator

// Comparison operators
#define DOT_SMALLER          (620) // the < operator
#define DOT_SMALLER_EQUAL    (621) // the <= operator
#define DOT_GREATER          (622) // the > operator
#define DOT_GREATER_EQUAL    (623) // the >= operator
#define DOT_EQUAL            (624) // the == operator
#define DOT_NOT_EQUAL        (625) // the != operator

// Bitwise operators
#define DOT_BIT_AND          (640) // the & operator
#define DOT_BIT_OR           (641) // the | operator
#define DOT_BIT_XOR          (642) // the ^ operator
#define DOT_BIT_NOT          (643) // the ~ operator
#define DOT_BIT_LEFT         (644) // the << operator
#define DOT_BIT_RIGHT        (645) // the >> operator

// General operators
#define DOT_COLON            (660) // the : operator
#define DOT_SEMI_COLON       (661) // the ; delimiter
#define DOT_COMMA            (662) // the , delimiter
#define DOT_BLOCK_OPEN       (663) // the { delimiter
#define DOT_BLOCK_CLOSE      (664) // the } delimiter
#define DOT_RBRACKET_OPEN    (665) // the ( delimiter
#define DOT_RBRACKET_CLOSE   (666) // the ) delimiter
#define DOT_SBRACKET_OPEN    (667) // the [ delimiter
#define DOT_SBRACKET_CLOSE   (668) // the ] delimiter
#define DOT_DEREFERNCE       (669) // the -> operator
#define DOT_FIELD            (670) // the . operator
#define DOT_CONDITION        (671) // the ? operator
#define DOT_SIZEOF           (672) // the sizeof operator
#define DOT_ADDRESSOF        (DOT_BIT_AND) // the & operator
#define DOT_VAR_ARG          (673) // the ... operator
#define DOT_PCONCAT          (674) // the ## operator
#define DOT_PSTRINGIFY       (675) // the # operator

// Assignment operators
#define DOT_ASSIGNMENT       (700) // the = operator
#define DOT_PLUS_ASSIGNMENT  (701) // the += operator
#define DOT_MINUS_ASSIGNMENT (702) // the -= operator
#define DOT_MULT_ASSIGNMENT  (703) // the *= operator
#define DOT_DIV_ASSIGNMENT   (704) // the /= operator
#define DOT_REM_ASSIGNMENT   (705) // the %= operator
#define DOT_LEFT_ASSIGNMENT  (706) // the <<= operator
#define DOT_RIGHT_ASSIGNMENT (707) // the >>= operator
#define DOT_AND_ASSIGNMENT   (708) // the &= operator
#define DOT_XOR_ASSIGNMENT   (709) // the ^= operator
#define DOT_OR_ASSIGNMENT    (710) // the |= operator

// Arithmetic operators
#define DOT_INCREMENT        (720) // the ++ operator
#define DOT_DECREMENT        (721) // the -- operator
#define DOT_PLUS             (722) // the + operator
#define DOT_MINUS            (723) // the - operator
#define DOT_STAR             (724) // the * operator
#define DOT_VALUEOF          (725) // the * operator
#define DOT_DIVISION         (726) // the / operator
#define DOT_REMAINDER        (727) // the % operator



//
// The class DObjcTokenizer implements an Objective-C lexical scanner. The 
// tokenizer scans a text stream for whitespace, strings, keywords, 
// comments, operators, numbers, predefined types, names, character constants,
// etc. There are two methods: nextToken and checkToken. The first method checks
// for a token and if there is a match, the scanner location is moved to
// the next location in the source. The second method only returns 
// the result of the match, the client must call the 'next' method to move the 
// scanner location.
//

@interface DObjcTokenizer : Object
{
@private
  DList       *_sources;        // the stack with the sources
  DSource     *_source;         // the source for the tokenizer
  
  DText       *_text;           // the last scanned, processed text
  DText       *_raw;            // the last scanned, unprocessed text
  BOOL         _eoff;           // is end of file reached ?
  BOOL         _eofl;           // is end of line reached ?
  BOOL         _sofl;           // is start of line active ?
  BOOL         _comment;        // is (block) comment active ?
  
  unsigned     _scanned;        // the length of the scanned text

  BOOL         _skipWhiteSpace; // should whitespace be skipped ? (def. YES)
  
  const
  char        *_scanLine;       // the current line for the scanner
  int          _currIndex;      // the current index in the scanner line
  int          _nextIndex;      // the next index in the scanner line
  int          _thrdIndex;      // the third index in the scanner line
  char         _currChar;       // the current character from the scanner line
  char         _nextChar;       // the next character from the scanner line
  char         _thrdChar;       // the third character from the scanner line
  BOOL         _currEsc;        // is the current character escaped ?
  BOOL         _nextEsc;        // is the next character escaped ?
  BOOL         _thrdEsc;        // is the third character escaped ?
}
#endif



@implementation DObjcTokenizer

#if _PRIVATE_

typedef struct
{
  char *name;
  int   token;
} _LookupTable;

_LookupTable _directives[] =
{
  { "assert",   DOT_PASSERT   },
  { "define",   DOT_PDEFINE   },
  { "elif",     DOT_PELIF     },
  { "else",     DOT_PELSE     },
  { "endif",    DOT_PENDIF    },
  { "error",    DOT_PERROR    },
  { "ident",    DOT_PIDENT    },
  { "if",       DOT_PIF       },
  { "ifdef",    DOT_PIFDEF    },
  { "ifndef",   DOT_PIFNDEF   },
  { "include",  DOT_PINCLUDE  },
  { "import",   DOT_PIMPORT   },
  { "line",     DOT_PLINE     },
  { "pragma",   DOT_PPRAGMA   },
  { "unassert", DOT_PUNASSERT },
  { "undef",    DOT_PUNDEF    },
  { "warning",  DOT_PWARNING  }
};

_LookupTable _keywords[] =
{
  { "char",     DOT_TCHAR    },
  { "double",   DOT_TDOUBLE  },
  { "float",    DOT_TFLOAT   },
  { "int",      DOT_TINT     },
  { "short",    DOT_TSHORT   },
  { "long",     DOT_TLONG    },
  { "unsigned", DOT_TUNSIGNED},
  { "auto",     DOT_AUTO     },
  { "const",    DOT_CONST    },
  { "extern",   DOT_EXTERN   },
  { "register", DOT_REGISTER },
  { "static",   DOT_STATIC   },
  { "enum",     DOT_ENUM     },
  { "struct",   DOT_STRUCT   },
  { "typedef",  DOT_TYPEDEF  },
  { "union",    DOT_UNION    },
  { "break",    DOT_BREAK    },
  { "case",     DOT_CASE     },
  { "continue", DOT_CONTINUE },
  { "default",  DOT_DEFAULT  },
  { "do",       DOT_DO       },
  { "else",     DOT_ELSE     },
  { "for",      DOT_FOR      },
  { "goto",     DOT_GOTO     },
  { "if",       DOT_IF       },
  { "return",   DOT_RETURN   },
  { "switch",   DOT_SWITCH   },
  { "volatile", DOT_VOLATILE },
  { "while",    DOT_WHILE    },
  { "sizeof",   DOT_SIZEOF   },
  { "void",     DOT_TVOID    },
  { "signed",   DOT_TSIGNED  }
};

_LookupTable _objcKeywords[] =
{
  {"interface",      DOT_INTERFACE     },
  {"implementation", DOT_IMPLEMENTATION},
  {"protocol",       DOT_PROTOCOL      },
  {"end",            DOT_END           },
  {"private",        DOT_PRIVATE       },
  {"protected",      DOT_PROTECTED     },
  {"public",         DOT_PUBLIC        },
  {"selector",       DOT_SELECTOR      },
  {"class",          DOT_CLASS         },
  {"encode",         DOT_ENCODE        },
  {"defs",           DOT_DEFS          },
  {"try",            DOT_TRY           },
  {"catch",          DOT_CATCH         },
  {"finally",        DOT_FINALLY       },
  {"throw",          DOT_THROW         },
  {"synchronized",   DOT_SYNCHRONIZED  },
  {"synchronize",    DOT_SYNCHRONIZE   }
};

_LookupTable _operators[] =
{
  { "&&",  DOT_AND             },
  { "&=",  DOT_AND_ASSIGNMENT  },
  { "&",   DOT_BIT_AND         },
  { "<=",  DOT_SMALLER_EQUAL   },
  { "<<",  DOT_BIT_LEFT        },
  { "<",   DOT_SMALLER         },
  { ">=",  DOT_GREATER_EQUAL   },
  { ">>",  DOT_BIT_RIGHT       },
  { ">",   DOT_GREATER         },
  { "==",  DOT_EQUAL           },
  { "=",   DOT_ASSIGNMENT      },
  { "!=",  DOT_NOT_EQUAL       },
  { "!",   DOT_NOT             },
  { "||",  DOT_OR              },
  { "|=",  DOT_OR_ASSIGNMENT   },
  { "|",   DOT_BIT_OR          },
  { "^",   DOT_BIT_XOR         },
  { "^=",  DOT_XOR_ASSIGNMENT  },
  { "*=",  DOT_MULT_ASSIGNMENT },
  { "*",   DOT_STAR            },
  { "/=",  DOT_DIV_ASSIGNMENT  }, 
  { "/",   DOT_DIVISION        },
  { "%=",  DOT_REM_ASSIGNMENT  },
  { "%",   DOT_REMAINDER       },
  { "<<=", DOT_LEFT_ASSIGNMENT },
  { ">>=", DOT_RIGHT_ASSIGNMENT},
  { "++",  DOT_INCREMENT       },
  { "+=",  DOT_PLUS_ASSIGNMENT },
  { "+",   DOT_PLUS            },
  { "-=",  DOT_MINUS_ASSIGNMENT},
  { "--",  DOT_DECREMENT       },
  { "->",  DOT_DEREFERNCE      },
  { "-",   DOT_MINUS           },
  { "~",   DOT_BIT_NOT         },
  { ":",   DOT_COLON           },
  { ";",   DOT_SEMI_COLON      },
  { ",",   DOT_COMMA           },
  { "{",   DOT_BLOCK_OPEN      },
  { "}",   DOT_BLOCK_CLOSE     },
  { "(",   DOT_RBRACKET_OPEN   },
  { ")",   DOT_RBRACKET_CLOSE  },
  { "[",   DOT_SBRACKET_OPEN   },
  { "]",   DOT_SBRACKET_CLOSE  },
  { ".",   DOT_FIELD           },
  { "...", DOT_VAR_ARG         },
  { "?",   DOT_CONDITION       },
  { "##",  DOT_PCONCAT         },
  { "#",   DOT_PSTRINGIFY      }
};
#endif


//// Constructors

//
// Initialise a tokenizer
//
// @return the object
//
- (DObjcTokenizer *) init
{
  [super init];

  _sources        = [DList new];
  _source         = nil;
   
  _text           = [DText new];
  _raw            = [DText new];
  _eoff           = NO;
  _eofl           = NO;
  _sofl           = NO;
  _comment        = NO;
  
  _scanned        = 0;
  
  _skipWhiteSpace = YES;

  _scanLine       = NULL;
  _currIndex      = 0;
  _nextIndex      = 0;
  _thrdIndex      = 0;
  _currChar       = EOS;
  _nextChar       = EOS;
  _thrdChar       = EOS;
  _currEsc        = NO;
  _nextEsc        = NO;
  _thrdEsc        = NO;
  
  return self;
}

//
// Initialise a tokenizer with a file
//
// @param source   the source
// @param name     the name for the source
// 
// @return the object
//
- (DObjcTokenizer *) init :(id <DTextReadable>) source :(const char *) name
{
  [self init];
  
  [self source :source :name];
  
  return self;
}

//// Copy related methods

// 
// Deepen a copy of the object (not implemented)
//
// @return the object
//
- deepen
{
  [super deepen];

  WARNING(DW_METHOD_NOT_IMPL, "deepen");

  return self;
}


//// Deconstructor

//
// Free the tokenizer (but *NOT* the TextReadables)
//
// @return the object
//
- free
{
  [_sources free];

  [_text free];
  [_raw  free];
  
  if (_source != nil)
    [_source free];

  return [super free];
}


//// Class methods

//
// Return the text for an operator
// 
// @param token     the operator token
// 
// @return the text for the operator (or NULL)
// 
+ (const char *) operator :(int) token
{
  int index = 0;
  
  while ((index < (sizeof(_operators) / sizeof(_operators[0]))) && (_operators[index].token != token))
  {
    index++;
  }
  
  if (index < (sizeof(_operators) / sizeof(_operators[0])))
    return _operators[index].name;
  else
    return NULL;
}


//
// Check if the token is a logical operator token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isLogicalOperator :(int) token
{
  return ((token >= DOT_AND) && (token <= DOT_NOT));
}

//
// Check if the token is a comparison operator token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isComparisonOperator :(int) token
{
  return ((token >= DOT_SMALLER) && (token <= DOT_NOT_EQUAL));
}

//
// Check if the token is a logical operator token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isBitOperator :(int) token
{
  return ((token >= DOT_BIT_AND) && (token <= DOT_BIT_RIGHT));
}

//
// Check if the token is an assignment operator token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isAssignmentOperator :(int) token
{
  return ((token >= DOT_ASSIGNMENT) && (token <= DOT_OR_ASSIGNMENT));
}

//
// Check if the token is an assignment operator token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isArithmeticOperator :(int) token
{
  return ((token >= DOT_INCREMENT) && (token <= DOT_REMAINDER));
}

//
// Check if the token is an operator token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isOperator :(int) token
{
  return (((token >= DOT_COLON) && (token <= DOT_PSTRINGIFY)) ||
          ([DObjcTokenizer isArithmeticOperator :token] ||
           [DObjcTokenizer isAssignmentOperator :token] ||
           [DObjcTokenizer isBitOperator        :token] ||
           [DObjcTokenizer isComparisonOperator :token] ||
           [DObjcTokenizer isLogicalOperator    :token]));
}

//
// Return the text for a keyword token
// 
// @param token     the keyword token
// 
// @return the text for the keyword (or NULL)
// 
+ (const char *) keyword :(int) token
{
  int index = 0;
  
  while ((index < (sizeof(_keywords) / sizeof(_keywords[0]))) && (_keywords[index].token != token))
  {
    index++;
  }
  
  if (index < (sizeof(_keywords) / sizeof(_keywords[0])))
    return _keywords[index].name;
  else
    return NULL;
}

//
// Return the text for an Objective-c keyword
// 
// @param token   the keyword
// 
// @return the text for the keyword (or NULL)
// 
+ (const char *) objcKeyword :(int) token
{
  int index = 0;
  
  while ((index < (sizeof(_objcKeywords) / sizeof(_objcKeywords[0]))) && (_objcKeywords[index].token != token))
  {
    index++;
  }
  
  if (index < (sizeof(_objcKeywords) / sizeof(_objcKeywords[0])))
    return _objcKeywords[index].name;
  else
    return NULL;
}

//
// Check if the token is a predefined type keyword token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isPredefinedTypeKeyword :(int) token
{
  return ((token >= DOT_TCHAR) && (token <= DOT_TSIGNED));
}

//
// Check if the token is a storage keyword token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isStorageKeyword :(int) token
{
  return ((token >= DOT_AUTO) && (token <= DOT_VOLATILE));
}

//
// Check if the token is a type definition keyword token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isTypeDefinitionKeyword :(int) token
{
  return ((token >= DOT_ENUM) && (token <= DOT_UNION));
}

//
// Check if the token is a flow keyword token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isFlowKeyword :(int) token
{
  return ((token >= DOT_BREAK) && (token <= DOT_WHILE));
}

//
// Check if the token is a keyword token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isKeyword :(int) token
{
  return ([DObjcTokenizer isFlowKeyword           :token] ||
          [DObjcTokenizer isTypeDefinitionKeyword :token] ||
          [DObjcTokenizer isStorageKeyword        :token] ||
          [DObjcTokenizer isPredefinedTypeKeyword :token]);
}

//
// Check if the token is an objective-c keyword token
// 
// @param token     the token
// 
// @return is it?
// 
+ (BOOL) isObjcKeyword :(int) token
{
  return ((token >= DOT_INTERFACE) && (token <= DOT_SYNCHRONIZE));
}

//
// Return the text for a (preprocessor) directive token
//
// @param token     the directive token
// 
// @return the text for the directive (or NULL)
// 
+ (const char *) directive :(int) token
{
  int index = 0;

  while ((index < (sizeof(_directives) / sizeof(_directives[0]))) && (_directives[index].token != token))
  {
    index++;
  }
  
  if (index < (sizeof(_directives) / sizeof(_directives[0])))
    return _directives[index].name;
  else
    return NULL;
}

//
// Check if the token is a (preprocessor) directive token
//
// @param token     the token
// 
// @return is it ?
// 
+ (BOOL) isDirective :(int) token
{
  return ((token >= DOT_PASSERT) && (token <= DOT_PWARNING));
}


//// Member methods

//
// Get the last scanned and processed text
// 
// @return the last scanned text (or NULL)
// 
- (const char *) text
{
  return ([_text cstring]);
}

//
// Get the last scanned and unprocessed text
// 
// @return the last unprocessed text (or NULL)
// 
- (const char *) raw
{
  return [_raw cstring];
}

//
// Get the name of the current file
// 
// @return the name of the file (or NULL)
// 
- (const char *) name
{
  return ((_source != nil) ? [_source name] : NULL);
}
   
//
// Get the current line number
//
// @return the current line number
//
- (int) lineNumber
{
  return ((_source != nil) ? [_source lineNumber] : 0);
}

//
// Get the current column number
//
// @return the current column number
//
- (int) columnNumber
{
  return ((_source != nil) ? [_source columnNumber] : 0);
}

//
// Is the end of file reached ?
// 
// @return is it ?
// 
- (BOOL) isEof
{
  return _eoff;
}

//
// Return whether white space is be skipped 
// 
// @return is it ?
// 
- (BOOL) skipWhiteSpace
{
  return _skipWhiteSpace;
}

//
// Set whether white space should be skipped
// 
// @param skip          should white space be skipped
// 
// @return the object
// 
- (DObjcTokenizer *) skipWhiteSpace :(BOOL) skip
{
  _skipWhiteSpace = skip;
  
  return self;
}


//// Source methods

//
// Start using a source (an already open source is pushed on the sources stack)
// 
// @param source   the source
// @param name     the name for the source
// 
// @return success
// 
- (BOOL) source :(id <DTextReadable>) source :(const char *) name
{
  if (_source != nil)
  {
    [_sources append :source];
  }

  _source = [DSource new];
  
  _sofl = YES;
  
  return [_source set :source :name];
}

//
// Pop a source from the sources stack
// 
// @return success
// 
- (BOOL) popSource
{
  BOOL ok = NO;
  
  if ([_sources length] > 0)
  {
    if (_source != nil)
      [_source free];
    
    _source = [_sources pop];
    
    ok = YES;
  }
  
  return ok;
}


//// Scanner methods

//
// Scan the source for the next token (white space is skipped if
// this is active)
// 
// @return the next token
// 
- (int) nextToken 
{
  int token;
  
  do
  {
    token = [self checkToken];
  
    if (token != DOT_EOFF)
    {
      [self next];
    }
  }
  while ((token == DOT_WHITESPACE) && (_skipWhiteSpace));

  return token;
}

//
// Check the source for the next token (white space is not skipped)
//
// @return the scanned token
// 
- (int) checkToken
{
  int token = DOT_EOFF;

  _scanned = 0;
  
  [_text clear];

  if (_eofl)
  {
    token = DOT_EOFL; // End Of Line
      
    [_text push :'\n'];
      
    _eofl = NO;
    _sofl = YES;
  }
  else if ((_source != nil) && (!_eoff))
  {
    [self _scanInit];

    if (_comment) // Block comment
    {
      while ((_currChar != EOS) && ((_currChar != '*') || (_nextChar != '/')))
      {
        [self _scanNext];
      }
      
      if (_currChar != EOS)
      {
        [self _scanNext];
        
        _comment = NO;
      }
    }
    else if (isspace(_currChar))
    {
      token = DOT_WHITESPACE; // Whitespace
      
      while (isspace(_nextChar))
      {
        [self _scanNext];
      }
    }
    else if ((_currChar == '/') && (_nextChar == '*'))
    {
      token = DOT_COMMENT; // Block comment
      
      [self _scanNext];
      [self _scanNext];
      
      while ((_currChar != '*') && (_nextChar != '/') && (_currChar != EOS))
      {
        [self _scanNext];
      }
      
      if (_currChar == EOS)
      {
        _comment = YES;
      }
      else
      {
        [self _scanNext];
      }
    }
    else if ((_currChar == '/') && (_nextChar == '/'))
    {
      token = DOT_COMMENT; // Line comment
      
      [self _scanNext];
      [self _scanNext];
      
      while (_currChar != EOS)
      {
        [self _scanNext];
      }
    }
    else if (_currChar == '#')
    {
      if (_sofl)
      {
        DText *directive = [DText new]; // Preprocessor directives
      
        while (isspace(_nextChar))
        {
          [self _scanNext];
        }
      
        while (isalpha(_nextChar))
        {
          [self _scanNext];
        
          [ directive push :_currChar];
        }
      
        token = DOT_PREPROCESSOR;
      
        {
          int index = 0;
        
          while ((index < (sizeof(_directives) / sizeof(_directives[0]))) && ([directive ccompare :_directives[index].name] != 0))
          {
            index++;
          }
        
          if (index < (sizeof(_directives) / sizeof(_directives[0])))
          {
            token = _directives[index].token;
          }
        }
        [directive free];
        
        _sofl = NO;
      }
    }
    else
    {
      _sofl = NO;
      
      if (_currChar == '@')
      {
        DText *keyword = [DText new]; // Objective-c keywords and strings
      
        while (isspace(_nextChar))
        {
          [self _scanNext];
        }

        if (_nextChar == '"')
        {
          token = DOT_OBJC_STRING;
      
          do
          {
            [self _scanNext];
          }
          while ((_nextChar != '"') || (_nextEsc));
      
          [self _scanNext];
        }
        else
        {
          while (isalpha(_nextChar))
          {
            [self _scanNext];
        
            [ keyword push :_currChar];
          }
      
          token = DOT_UNKNOWN;
      
          {
            int index = 0;
        
            while ((index < (sizeof(_objcKeywords) / sizeof(_objcKeywords[0]))) && ([keyword ccompare :_objcKeywords[index].name] != 0))
            {
              index++;
            }
        
            if (index < (sizeof(_objcKeywords) / sizeof(_objcKeywords[0])))
            {
              token = _objcKeywords[index].token;
            }
          }
          [keyword free];
        }
      }
      else if (_currChar == '"')
      {
        token = DOT_STRING; // String constant
        
        do
        {
          [self _scanNext];
        }
        while ((_currChar != '"') || (_currEsc));
      }
      else if ((_currChar == 'L') && (_nextChar == '"'))
      {
        token = DOT_WIDE_STRING; // Wide string constant
        
        [self _scanNext];
        
        do
        {
          [self _scanNext];
        }
        while ((_currChar != '"') || (_currEsc));
      }
      else if (_currChar == '\'')
      {
        token = DOT_CHAR; // Character constant
        
        do
        {
          [self _scanNext];
        }
        while ((_currChar != '\'') || (_currEsc));
      }
      else if ((_currChar == 'L') && (_nextChar == '\''))
      {
        token = DOT_WIDE_CHAR; // Wide Character constant

        [self _scanNext];
        do
        {
          [self _scanNext];
        }
        while ((_currChar != '\'') || (_currEsc));
      }
      else if ((isdigit(_currChar)) || ((_currChar == '.') && (isdigit(_nextChar))))
      {
        token = [self _scanNumber]; // Number constant
      }
      else if ((isalpha(_currChar)) || (_currChar == '_'))
      {
        token = DOT_IDENTIFIER; // Identifier

        while ((isalpha(_nextChar)) || (isdigit(_nextChar)) || (_nextChar == '_'))
        {
          [self _scanNext];
        }
        
        { 
          int   index = 0;
          const 
          char *keyword = [_text cstring];
        
          while ((index < (sizeof(_keywords) / sizeof(_keywords[0]))) && (strcmp(_keywords[index].name, keyword) != 0))
          {
            index++;
          }
        
          if (index < (sizeof(_keywords) / sizeof(_keywords[0])))
          {
            token = _keywords[index].token;
          }
        }
      }
      else // Operator
      {
        int index;
        int match1  = -1; // match one character
        int match2  = -1; // match two character
        int match3  = -1; // match tree characters
        
        for (index = 0; index < sizeof(_operators) / sizeof(_operators[0]); index++)
        {
          if (_operators[index].name[0] == _currChar)
          {
            if (_operators[index].name[1] == EOS)
            {
              // Single character operator
              match1 = _operators[index].token;
            }
            else if (_operators[index].name[1] == _nextChar)
            {
              if (_operators[index].name[2] == EOS)
              {
                // Two character operator
                match2 = _operators[index].token;
              }
              else if ((_operators[index].name[2] == _thrdChar) && (_operators[index].name[3] == EOS))
              {
                // Three character operator
                match3 = _operators[index].token;
              }
            }
          }
        }
        
        if (match3 >= 0)
        {
          token = match3;
          
          [self _scanNext];
          [self _scanNext];
        }
        else if (match2 >= 0)
        {
          token = match2;
          
          [self _scanNext];
        }
        else if (match1 >= 0)
        {
          token = match1;
        }
        else
        {
          token = DOT_UNKNOWN;
        }
      }
    }
    
    _scanned = self->_currIndex;
  }

  return (token);
}

//
// Move the scanned indicator after a checkToken method
// 
// @return the object
// 
- (DObjcTokenizer *) next 
{
  if (_scanned > 0)
  {
    [self _next :_scanned];
    
    _scanned = 0;
  }

  return self;
}
    

//// Error methods

//
// Generate an error on stderr for the current file and line
// 
// @param msg      the error message
// 
// @return the object
// 
- (DObjcTokenizer *) error :(const char *) msg
{
  if (_source == nil)
  {
    WARNING(DW_OBJECT_NOT_INIT, "source");
  }
  else
  {
    [_source error :msg];
  }
  
  return self;
}


//// Basic scanner methods (private methods)

//
// Scan the next character
// 
// @return the scanned character
// 
- (char) _scanNext
{
  char  ch;
  
  _currIndex = _nextIndex;
  _currChar  = _nextChar;
  _currEsc   = _nextEsc;

  _nextIndex = _thrdIndex;
  _nextChar  = _thrdChar;
  _nextEsc   = _thrdEsc;
  
  _thrdEsc   = NO;
  
  if ((_thrdChar != EOS) || (_thrdIndex == 0))
  {
    ch = _scanLine[_thrdIndex++];

    if (ch != EOS)
    {
      [_raw push :ch];
    }
    
    while ((!_eoff) && (ch == '\\') && (_scanLine[_thrdIndex] == EOS)) // concatted lines
    {
      if (![_source appendLine])
      {
        _eoff = YES; // no concatted lines over files..
      }
      else
      {
        _scanLine  = [_source line];
        
        ch = _scanLine[_thrdIndex++];
      }
    }
 
    if ((ch == '?') && (_scanLine[_thrdIndex] == '?'))
    {
      // trigraphs
      switch(_scanLine[_thrdIndex+1])
      {
       case '(' : ch = '[';  break;
       case '/' : ch = '\\'; break;
       case ')' : ch = ']';  break;
       case '\'': ch = '^';  break;
       case '<' : ch = '{';  break;
       case '!' : ch = '|';  break;
       case '>' : ch = '}';  break;
       case '-' : ch = '~';  break;
       case '=' : ch = '#';   break;
      }
      
      if (ch != '?')
      {
        [_raw push :_scanLine[_thrdIndex++]];
        [_raw push :_scanLine[_thrdIndex++]];
      }
    }
    else
    {
      // digraphs
      char next = _scanLine[_thrdIndex++];
      
      if ((ch == '<') && (next == ':'))
      {
        ch = '['; [_raw push :next];
      }
      else if ((ch == ':') && (next == '>'))
      {
        ch = ']'; [_raw push :next];
      }
      else if ((ch == '<') && (next == '%'))
      {
        ch = '{'; [_raw push :next];
      }
      else if ((ch == '%') && (next == '>'))
      {
        ch = '}'; [_raw push :next];
      }
      else if ((ch == '%') && (next == ':'))
      {
        ch = '#'; [_raw push :next];
      }
      else
        _thrdIndex--;
    }
    
    if (ch == '\\')
    {
      _thrdEsc = YES;
    
      if (_scanLine[_thrdIndex] == 'x')
      {
        // hex escape sequence
        [_raw push :_scanLine[_thrdIndex++]];
      
        ch = 0;
        while (isxdigit(_scanLine[_thrdIndex]))
        {
          ch *= 16;
        
          if ((_scanLine[_thrdIndex] >= '0') && (_scanLine[_thrdIndex] <= '9'))
            ch += (_scanLine[_thrdIndex] - '0');
          else if ((_scanLine[_thrdIndex] >= 'a') && (_scanLine[_thrdIndex] <= 'f'))
            ch += (_scanLine[_thrdIndex] - 'a') + 10;
          else
            ch += (_scanLine[_thrdIndex] - 'A') + 10;
        
          [_raw push :_scanLine[_thrdIndex++]];
        }
      }
      else if (_scanLine[_thrdIndex] == '0')
      {
        // octal escape sequence
        int cnt = 0;
      
        ch = 0;
      
        [_raw push :_scanLine[_thrdIndex++]];
        
        while ((cnt < 3) && (_scanLine[_thrdIndex] >= '0') && (_scanLine[_thrdIndex] <= '7'))
        {
          ch *= 8;
          ch += (_scanLine[_thrdIndex] - '0');
        
          [_raw push :_scanLine[_thrdIndex++]];
          cnt--;
        }
      }
      else
      {
        // remaining escape sequences
        switch(_scanLine[_thrdIndex])
        {
         case 'a' : ch = '\a'; break;
         case 'b' : ch = '\b'; break;
         case 'f' : ch = '\f'; break;
         case 'n' : ch = '\n'; break;
         case 'r' : ch = '\r'; break;
         case 't' : ch = '\t'; break;
         case 'v' : ch = '\v'; break;
         default  : ch = _scanLine[_thrdIndex]; break;
        }
        [_raw push :_scanLine[_thrdIndex++]];
      }
    }
  }
  else
    ch = EOS;
  
  if (_currChar != EOS)
    [_text push :_currChar];
  
  _thrdChar  = ch;
  
  return _currChar;
}

//
// Initialise the low level scanner
// 
// @return the (first) scanned character
// 
- (char) _scanInit
{
  _scanLine  = [_source line];
  _currIndex = -1;
  _nextIndex = -1;
  _thrdIndex = 0;
  _currChar  = EOS;
  _nextChar  = EOS;
  _thrdChar  = EOS;
  _currEsc   = NO;
  _nextEsc   = NO;
  _thrdEsc   = NO;
  
  [self _scanNext];
  [self _scanNext];
  [self _scanNext];
  
  return _currChar;
}

//
// Move the scanned indicator a number of positions (private)
//
// @param positions  the number of position to move
//  
// @return the object
// 
- (DObjcTokenizer *) _next :(unsigned) positions
{
  if (_source != nil)
  {
    if ([_source scanned :positions])
    {
      _eofl = YES;
      _sofl = YES;
      
      if (![_source nextLine])
      {
        if (![self popSource])
        {
          _eoff = YES;
        }
      }
    }
  }
  
  return self;
}

//
// Scan for a number
// 
// @return the token 
// 
- (int) _scanNumber
{
  int token = DOT_UNKNOWN;

  if (isdigit(_currChar))
  {
    if (_currChar == '0')
    {
      if (toupper(_nextChar) == 'X')
      {
        token = DOT_HEX_NUMBER;

        [self _scanNext];
      
        while (isxdigit(_nextChar))
        {
          [self _scanNext];
        }
      }
      else
      {
        token = DOT_OCT_NUMBER;

        while (isdigit(_nextChar))
        {
          [self _scanNext];
        
          if (_currChar > '7') // not an octal number, must be floating point ..
          {
            token = DOT_FLP_NUMBER;
          }
        }
      }
    }
    else if (isdigit(_currChar))
    {
      token = DOT_DEC_NUMBER;
    
      while (isdigit(_nextChar))
      {
        [self _scanNext];
      }
    }
    
    if ((_nextChar == '.') && (token != DOT_HEX_NUMBER))
    {
      token = DOT_FLP_NUMBER;
    
      [self _scanNext];
    }
  }
  else if (_currChar == '.')
  {
    token = DOT_FLP_NUMBER;
  }
  
  
  if (token == DOT_FLP_NUMBER)
  {
    while (isdigit(_nextChar))
    {
      [self _scanNext];
    }
  }
  
  if ((toupper(_nextChar) == 'E') && ((token = DOT_FLP_NUMBER) || (token == DOT_DEC_NUMBER) || (token == DOT_OCT_NUMBER)))
  {
    token = DOT_FLP_NUMBER;
    
    [self _scanNext]; 
  
    if ((_nextChar == '+') || (_nextChar == '-'))
    {
      [self _scanNext];
    }
    
    while (isdigit(_nextChar))
    {
      [self _scanNext];
    }
  }
      
  if ((token == DOT_DEC_NUMBER) || (token == DOT_HEX_NUMBER) || (token == DOT_OCT_NUMBER))
  {
    if ((toupper(_nextChar) == 'L') || (toupper(_nextChar) == 'U'))
    {
      char prev = toupper(_nextChar);
      
      [self _scanNext];
      
      if (((toupper(_nextChar) == 'L') || (toupper(_nextChar) == 'U')) && (toupper(_nextChar) != prev))
      {
        [self _scanNext];
      }
    }
  }
  else if (token == DOT_FLP_NUMBER)
  {
    if ((toupper(_nextChar) == 'F') || (toupper(_nextChar) == 'L'))
    {
      [self _scanNext];
    }
  }

  return token;
}
@end

/*==========================================================================*/
